package cdc.retrofit.official.crawler;

import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import okhttp3.ConnectionPool;
import okhttp3.Dispatcher;
import okhttp3.HttpUrl;
import okhttp3.OkHttpClient;
import retrofit2.Call;
import retrofit2.Callback;
import retrofit2.Response;
import retrofit2.Retrofit;

/**
 * 使用Retrofit服务将网址转换为网页的简单网络抓取工具
 * 
 * @author cdc
 *
 */
public class Crawler {
	private final Set<HttpUrl> fetchedUrls = Collections.synchronizedSet(new LinkedHashSet<HttpUrl>());
	private final ConcurrentHashMap<String, AtomicInteger> hostnames = new ConcurrentHashMap<>();
	private final PageService pageService;

	public Crawler(PageService pageService) {
		this.pageService = pageService;
	}

	public void crawlPage(HttpUrl url) {
		// Skip hosts that we've visited many times.
		AtomicInteger hostnameCount = new AtomicInteger();
		AtomicInteger previous = hostnames.putIfAbsent(url.host(), hostnameCount);
		if (previous != null)
			hostnameCount = previous;
		if (hostnameCount.incrementAndGet() > 100)
			return;

		// Asynchronously visit URL.
		pageService.get(url).enqueue(new Callback<Page>() {
			@Override
			public void onResponse(Call<Page> call, Response<Page> response) {
				if (!response.isSuccessful()) {
					System.out.println(call.request().url() + ": failed: " + response.code());
					return;
				}

				// Print this page's URL and title.
				Page page = response.body();
				HttpUrl base = response.raw().request().url();
				System.out.println(base + ": " + page.title);

				// Enqueue its links for visiting.
				for (String link : page.links) {
					HttpUrl linkUrl = base.resolve(link);
					if (linkUrl != null && fetchedUrls.add(linkUrl)) {
						crawlPage(linkUrl);
					}
				}
			}

			@Override
			public void onFailure(Call<Page> call, Throwable t) {
				System.out.println(call.request().url() + ": failed: " + t);
			}
		});
	}

	public static void main(String... args) throws Exception {
		Dispatcher dispatcher = new Dispatcher(Executors.newFixedThreadPool(20));
		dispatcher.setMaxRequests(20);
		dispatcher.setMaxRequestsPerHost(1);

		OkHttpClient okHttpClient = new OkHttpClient.Builder().dispatcher(dispatcher)
				.connectionPool(new ConnectionPool(100, 30, TimeUnit.SECONDS)).build();

		Retrofit retrofit = new Retrofit.Builder().baseUrl(HttpUrl.parse("https://www.baidu.com/"))
				.addConverterFactory(PageAdapter.FACTORY).client(okHttpClient).build();

		PageService pageService = retrofit.create(PageService.class);

		Crawler crawler = new Crawler(pageService);
		String aa = "https://www.baidu.com/";
		crawler.crawlPage(HttpUrl.parse(aa));
	}

}
